{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cee26370",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Description</th>\n",
       "      <th>Amount</th>\n",
       "      <th>Transaction Type</th>\n",
       "      <th>category</th>\n",
       "      <th>Account Name</th>\n",
       "      <th>pass</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>03/16/2018</td>\n",
       "      <td>Biweekly Paycheck</td>\n",
       "      <td>2000.00</td>\n",
       "      <td>credit</td>\n",
       "      <td>Paycheck</td>\n",
       "      <td>Checking</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>03/17/2018</td>\n",
       "      <td>Brewing Company</td>\n",
       "      <td>19.50</td>\n",
       "      <td>debit</td>\n",
       "      <td>Alcohol &amp; Bars</td>\n",
       "      <td>Silver Card</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>03/17/2018</td>\n",
       "      <td>Pizza Place</td>\n",
       "      <td>23.34</td>\n",
       "      <td>debit</td>\n",
       "      <td>Fast Food</td>\n",
       "      <td>Platinum Card</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>03/19/2018</td>\n",
       "      <td>Mediterranean Restaurant</td>\n",
       "      <td>36.48</td>\n",
       "      <td>debit</td>\n",
       "      <td>Restaurants</td>\n",
       "      <td>Silver Card</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>03/19/2018</td>\n",
       "      <td>City Water Charges</td>\n",
       "      <td>35.00</td>\n",
       "      <td>debit</td>\n",
       "      <td>Utilities</td>\n",
       "      <td>Checking</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>801</th>\n",
       "      <td>07/19/2019</td>\n",
       "      <td>Mexican Restaurant</td>\n",
       "      <td>28.00</td>\n",
       "      <td>debit</td>\n",
       "      <td>Restaurants</td>\n",
       "      <td>Platinum Card</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>802</th>\n",
       "      <td>07/22/2019</td>\n",
       "      <td>Credit Card Payment</td>\n",
       "      <td>257.08</td>\n",
       "      <td>credit</td>\n",
       "      <td>Credit Card Payment</td>\n",
       "      <td>Silver Card</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>803</th>\n",
       "      <td>07/22/2019</td>\n",
       "      <td>Thai Restaurant</td>\n",
       "      <td>26.67</td>\n",
       "      <td>debit</td>\n",
       "      <td>Restaurants</td>\n",
       "      <td>Silver Card</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>804</th>\n",
       "      <td>07/23/2019</td>\n",
       "      <td>Credit Card Payment</td>\n",
       "      <td>257.08</td>\n",
       "      <td>debit</td>\n",
       "      <td>Credit Card Payment</td>\n",
       "      <td>Checking</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>805</th>\n",
       "      <td>07/24/2019</td>\n",
       "      <td>Starbucks</td>\n",
       "      <td>2.50</td>\n",
       "      <td>debit</td>\n",
       "      <td>Coffee Shops</td>\n",
       "      <td>Silver Card</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>806 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           Date               Description   Amount Transaction Type  \\\n",
       "0    03/16/2018         Biweekly Paycheck  2000.00           credit   \n",
       "1    03/17/2018           Brewing Company    19.50            debit   \n",
       "2    03/17/2018               Pizza Place    23.34            debit   \n",
       "3    03/19/2018  Mediterranean Restaurant    36.48            debit   \n",
       "4    03/19/2018        City Water Charges    35.00            debit   \n",
       "..          ...                       ...      ...              ...   \n",
       "801  07/19/2019        Mexican Restaurant    28.00            debit   \n",
       "802  07/22/2019       Credit Card Payment   257.08           credit   \n",
       "803  07/22/2019           Thai Restaurant    26.67            debit   \n",
       "804  07/23/2019       Credit Card Payment   257.08            debit   \n",
       "805  07/24/2019                 Starbucks     2.50            debit   \n",
       "\n",
       "                category   Account Name  pass  \n",
       "0               Paycheck       Checking  True  \n",
       "1         Alcohol & Bars    Silver Card   NaN  \n",
       "2              Fast Food  Platinum Card   NaN  \n",
       "3            Restaurants    Silver Card   NaN  \n",
       "4              Utilities       Checking   NaN  \n",
       "..                   ...            ...   ...  \n",
       "801          Restaurants  Platinum Card   NaN  \n",
       "802  Credit Card Payment    Silver Card   NaN  \n",
       "803          Restaurants    Silver Card   NaN  \n",
       "804  Credit Card Payment       Checking   NaN  \n",
       "805         Coffee Shops    Silver Card   NaN  \n",
       "\n",
       "[806 rows x 7 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv(\"data/personal_transactions.csv\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "28d412b9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/rchaves/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Example({'_index': 0, 'description': 'Biweekly Paycheck', 'category': 'Paycheck'}) (input_keys={'category', 'description'})"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "\n",
    "sys.path.append(\"..\")\n",
    "\n",
    "from langwatch_nlp.studio.dspy.patched_boostrap_few_shot import ExampleWithEntryMap\n",
    "\n",
    "trainset = [\n",
    "    ExampleWithEntryMap(\n",
    "        _index=index, description=x[\"Description\"], category=x[\"category\"]\n",
    "    ).with_inputs(\"description\", \"category\")\n",
    "    for (index, x) in df[0:20].iterrows()\n",
    "]\n",
    "testset = [\n",
    "    ExampleWithEntryMap(\n",
    "        _index=index, description=x[\"Description\"], category=x[\"category\"]\n",
    "    ).with_inputs(\"description\", \"category\")\n",
    "    for (index, x) in df[20:40].iterrows()\n",
    "]\n",
    "\n",
    "trainset_no_category_input = [\n",
    "    ExampleWithEntryMap(\n",
    "        _index=index, description=x[\"Description\"], category=x[\"category\"]\n",
    "    ).with_inputs(\"description\")\n",
    "    for (index, x) in df[0:20].iterrows()\n",
    "]\n",
    "testset_no_category_input = [\n",
    "    ExampleWithEntryMap(\n",
    "        _index=index, description=x[\"Description\"], category=x[\"category\"]\n",
    "    ).with_inputs(\"description\")\n",
    "    for (index, x) in df[20:40].iterrows()\n",
    "]\n",
    "\n",
    "trainset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "39d7ccd9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LangWatch API key is already set, if you want to login again, please call as langwatch.login(relogin=True)\n"
     ]
    }
   ],
   "source": [
    "import langwatch\n",
    "\n",
    "langwatch.login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ac8d82b5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-10-29 23:50:52,770 - langwatch.utils.initialization - INFO - Setting up LangWatch client...\n",
      "2025-10-29 23:50:52,771 - langwatch.client - INFO - Configuring OTLP exporter with endpoint: http://localhost:5560/api/otel/v1/traces\n",
      "2025-10-29 23:50:52,772 - langwatch.client - INFO - Registering atexit handler to flush tracer provider on exit\n",
      "2025-10-29 23:50:52,773 - langwatch.client - INFO - Successfully configured tracer provider with OTLP exporter\n",
      "2025-10-29 23:50:52,773 - langwatch.utils.initialization - INFO - LangWatch client setup complete\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025/10/29 23:50:52 WARNING dspy.primitives.module: Calling module.forward(...) on WorkflowModule directly is discouraged. Please use module(...) instead.\n",
      "2025/10/29 23:50:55 INFO dspy.teleprompt.mipro_optimizer_v2: \n",
      "RUNNING WITH THE FOLLOWING LIGHT AUTO RUN SETTINGS:\n",
      "num_trials: 10\n",
      "minibatch: False\n",
      "num_fewshot_candidates: 6\n",
      "num_instruct_candidates: 3\n",
      "valset size: 20\n",
      "\n",
      "2025/10/29 23:50:55 INFO dspy.teleprompt.mipro_optimizer_v2: \n",
      "==> STEP 1: BOOTSTRAP FEWSHOT EXAMPLES <==\n",
      "2025/10/29 23:50:55 INFO dspy.teleprompt.mipro_optimizer_v2: These will be used as few-shot example candidates for our program and for creating instructions.\n",
      "\n",
      "2025/10/29 23:50:55 INFO dspy.teleprompt.mipro_optimizer_v2: Bootstrapping N=6 sets of demonstrations...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction(\n",
      "    classify_transaction=Prediction(\n",
      "    category='Gas & Fuel'\n",
      "),\n",
      "    end={'output': 'Gas & Fuel'},\n",
      "    evaluations={'exact_match': EvaluationResultWithMetadata(status='processed', score=1.0, passed=True, label=None, details=None, inputs={'data': {'output': 'Gas & Fuel', 'expected_output': 'Gas & Fuel'}}, cost=None, duration=1)}\n",
      ") \n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "[LangWatch] Experiment initialized, run_id: ancient-opalescent-chinchilla\n",
      "[LangWatch] Open http://localhost:5560/inbox-narrator/experiments/personal-transactions-from-notebook?runIds=ancient-opalescent-chinchilla to track your DSPy training session live\n",
      "\n",
      "Bootstrapping set 1/6\n",
      "Bootstrapping set 2/6\n",
      "Bootstrapping set 3/6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▌     | 9/20 [00:16<00:19,  1.80s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 9 examples for up to 1 rounds, amounting to 9 attempts.\n",
      "Bootstrapping set 4/6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|███       | 6/20 [00:06<00:14,  1.04s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 6 examples for up to 1 rounds, amounting to 6 attempts.\n",
      "Bootstrapping set 5/6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|█▌        | 3/20 [00:04<00:22,  1.34s/it]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
      "\u001b[31mKeyboardInterrupt\u001b[39m                         Traceback (most recent call last)",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[4]\u001b[39m\u001b[32m, line 218\u001b[39m\n\u001b[32m    205\u001b[39m         langwatch.dspy.init(\n\u001b[32m    206\u001b[39m             experiment=\u001b[33m\"\u001b[39m\u001b[33mpersonal-transactions-from-notebook\u001b[39m\u001b[33m\"\u001b[39m, optimizer=optimizer\n\u001b[32m    207\u001b[39m         )\n\u001b[32m    209\u001b[39m         optimizer.compile(\n\u001b[32m    210\u001b[39m             module,\n\u001b[32m    211\u001b[39m             trainset=trainset_no_category_input,\n\u001b[32m   (...)\u001b[39m\u001b[32m    214\u001b[39m             max_labeled_demos=\u001b[32m16\u001b[39m,\n\u001b[32m    215\u001b[39m         )\n\u001b[32m--> \u001b[39m\u001b[32m218\u001b[39m \u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/python-sdk/src/langwatch/telemetry/tracing.py:549\u001b[39m, in \u001b[36mLangWatchTrace.__call__.<locals>.wrapper\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m    547\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m._clone() \u001b[38;5;28;01mas\u001b[39;00m trace:\n\u001b[32m    548\u001b[39m     trace._set_callee_input_information(func, *args, **kwargs)\n\u001b[32m--> \u001b[39m\u001b[32m549\u001b[39m     output = \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    550\u001b[39m     trace._set_callee_output_information(func, output)\n\u001b[32m    551\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m output\n",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[4]\u001b[39m\u001b[32m, line 209\u001b[39m, in \u001b[36mrun\u001b[39m\u001b[34m()\u001b[39m\n\u001b[32m    199\u001b[39m optimizer = dspy.MIPROv2(\n\u001b[32m    200\u001b[39m     metric=metric,\n\u001b[32m    201\u001b[39m     auto=\u001b[33m\"\u001b[39m\u001b[33mlight\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m    202\u001b[39m     num_threads=\u001b[32m24\u001b[39m,\n\u001b[32m    203\u001b[39m )\n\u001b[32m    205\u001b[39m langwatch.dspy.init(\n\u001b[32m    206\u001b[39m     experiment=\u001b[33m\"\u001b[39m\u001b[33mpersonal-transactions-from-notebook\u001b[39m\u001b[33m\"\u001b[39m, optimizer=optimizer\n\u001b[32m    207\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m209\u001b[39m \u001b[43moptimizer\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcompile\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m    210\u001b[39m \u001b[43m    \u001b[49m\u001b[43mmodule\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    211\u001b[39m \u001b[43m    \u001b[49m\u001b[43mtrainset\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtrainset_no_category_input\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    212\u001b[39m \u001b[43m    \u001b[49m\u001b[43mvalset\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtestset_no_category_input\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    213\u001b[39m \u001b[43m    \u001b[49m\u001b[43mmax_bootstrapped_demos\u001b[49m\u001b[43m=\u001b[49m\u001b[32;43m4\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m    214\u001b[39m \u001b[43m    \u001b[49m\u001b[43mmax_labeled_demos\u001b[49m\u001b[43m=\u001b[49m\u001b[32;43m16\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m    215\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/python-sdk/src/langwatch/dspy/__init__.py:629\u001b[39m, in \u001b[36mLangWatchTrackedMIPROv2.compile\u001b[39m\u001b[34m(self, student, **kwargs)\u001b[39m\n\u001b[32m    627\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mcompile\u001b[39m(\u001b[38;5;28mself\u001b[39m, student, **kwargs):\n\u001b[32m    628\u001b[39m     \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m._patch_print_and_evaluate():\n\u001b[32m--> \u001b[39m\u001b[32m629\u001b[39m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcompile\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstudent\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/dspy/teleprompt/mipro_optimizer_v2.py:182\u001b[39m, in \u001b[36mMIPROv2.compile\u001b[39m\u001b[34m(self, student, trainset, teacher, valset, num_trials, max_bootstrapped_demos, max_labeled_demos, seed, minibatch, minibatch_size, minibatch_full_eval_steps, program_aware_proposer, data_aware_proposer, view_data_batch_size, tip_aware_proposer, fewshot_aware_proposer, requires_permission_to_run, provide_traceback)\u001b[39m\n\u001b[32m    171\u001b[39m evaluate = Evaluate(\n\u001b[32m    172\u001b[39m     devset=valset,\n\u001b[32m    173\u001b[39m     metric=\u001b[38;5;28mself\u001b[39m.metric,\n\u001b[32m   (...)\u001b[39m\u001b[32m    178\u001b[39m     provide_traceback=provide_traceback,\n\u001b[32m    179\u001b[39m )\n\u001b[32m    181\u001b[39m \u001b[38;5;66;03m# Step 1: Bootstrap few-shot examples\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m182\u001b[39m demo_candidates = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_bootstrap_fewshot_examples\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprogram\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrainset\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mseed\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mteacher\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    184\u001b[39m \u001b[38;5;66;03m# Step 2: Propose instruction candidates\u001b[39;00m\n\u001b[32m    185\u001b[39m instruction_candidates = \u001b[38;5;28mself\u001b[39m._propose_instructions(\n\u001b[32m    186\u001b[39m     program,\n\u001b[32m    187\u001b[39m     trainset,\n\u001b[32m   (...)\u001b[39m\u001b[32m    193\u001b[39m     fewshot_aware_proposer,\n\u001b[32m    194\u001b[39m )\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/dspy/teleprompt/mipro_optimizer_v2.py:347\u001b[39m, in \u001b[36mMIPROv2._bootstrap_fewshot_examples\u001b[39m\u001b[34m(self, program, trainset, seed, teacher)\u001b[39m\n\u001b[32m    342\u001b[39m \u001b[38;5;66;03m# try:\u001b[39;00m\n\u001b[32m    343\u001b[39m effective_max_errors = (\n\u001b[32m    344\u001b[39m     \u001b[38;5;28mself\u001b[39m.max_errors \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.max_errors \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m dspy.settings.max_errors\n\u001b[32m    345\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m347\u001b[39m demo_candidates = \u001b[43mcreate_n_fewshot_demo_sets\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m    348\u001b[39m \u001b[43m    \u001b[49m\u001b[43mstudent\u001b[49m\u001b[43m=\u001b[49m\u001b[43mprogram\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    349\u001b[39m \u001b[43m    \u001b[49m\u001b[43mnum_candidate_sets\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mnum_fewshot_candidates\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    350\u001b[39m \u001b[43m    \u001b[49m\u001b[43mtrainset\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtrainset\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    351\u001b[39m \u001b[43m    \u001b[49m\u001b[43mmax_labeled_demos\u001b[49m\u001b[43m=\u001b[49m\u001b[43m(\u001b[49m\u001b[43mLABELED_FEWSHOT_EXAMPLES_IN_CONTEXT\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mzeroshot\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mmax_labeled_demos\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    352\u001b[39m \u001b[43m    \u001b[49m\u001b[43mmax_bootstrapped_demos\u001b[49m\u001b[43m=\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m    353\u001b[39m \u001b[43m        \u001b[49m\u001b[43mBOOTSTRAPPED_FEWSHOT_EXAMPLES_IN_CONTEXT\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mzeroshot\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mmax_bootstrapped_demos\u001b[49m\n\u001b[32m    354\u001b[39m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    355\u001b[39m \u001b[43m    \u001b[49m\u001b[43mmetric\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mmetric\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    356\u001b[39m \u001b[43m    \u001b[49m\u001b[43mmax_errors\u001b[49m\u001b[43m=\u001b[49m\u001b[43meffective_max_errors\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    357\u001b[39m \u001b[43m    \u001b[49m\u001b[43mteacher\u001b[49m\u001b[43m=\u001b[49m\u001b[43mteacher\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    358\u001b[39m \u001b[43m    \u001b[49m\u001b[43mteacher_settings\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mteacher_settings\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    359\u001b[39m \u001b[43m    \u001b[49m\u001b[43mseed\u001b[49m\u001b[43m=\u001b[49m\u001b[43mseed\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    360\u001b[39m \u001b[43m    \u001b[49m\u001b[43mmetric_threshold\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mmetric_threshold\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    361\u001b[39m \u001b[43m    \u001b[49m\u001b[43mrng\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mrng\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    362\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    363\u001b[39m \u001b[38;5;66;03m# NOTE: Bootstrapping is essential to MIPRO!\u001b[39;00m\n\u001b[32m    364\u001b[39m \u001b[38;5;66;03m# Failing silently here makes the rest of the optimization far weaker as a result!\u001b[39;00m\n\u001b[32m    365\u001b[39m \u001b[38;5;66;03m# except Exception as e:\u001b[39;00m\n\u001b[32m    366\u001b[39m \u001b[38;5;66;03m#     logger.info(f\"!!!!\\n\\n\\n\\n\\nError generating few-shot examples: {e}\")\u001b[39;00m\n\u001b[32m    367\u001b[39m \u001b[38;5;66;03m#     logger.info(\"Running without few-shot examples.!!!!\\n\\n\\n\\n\\n\")\u001b[39;00m\n\u001b[32m    368\u001b[39m \u001b[38;5;66;03m#     demo_candidates = None\u001b[39;00m\n\u001b[32m    370\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m demo_candidates\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/dspy/teleprompt/utils.py:408\u001b[39m, in \u001b[36mcreate_n_fewshot_demo_sets\u001b[39m\u001b[34m(student, num_candidate_sets, trainset, max_labeled_demos, max_bootstrapped_demos, metric, teacher_settings, max_errors, max_rounds, labeled_sample, min_num_samples, metric_threshold, teacher, include_non_bootstrapped, seed, rng)\u001b[39m\n\u001b[32m    396\u001b[39m     size = rng.randint(min_num_samples, max_bootstrapped_demos)\n\u001b[32m    398\u001b[39m     teleprompter = BootstrapFewShot(\n\u001b[32m    399\u001b[39m         metric=metric,\n\u001b[32m    400\u001b[39m         max_errors=max_errors,\n\u001b[32m   (...)\u001b[39m\u001b[32m    405\u001b[39m         max_rounds=max_rounds,\n\u001b[32m    406\u001b[39m     )\n\u001b[32m--> \u001b[39m\u001b[32m408\u001b[39m     program2 = \u001b[43mteleprompter\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcompile\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m    409\u001b[39m \u001b[43m        \u001b[49m\u001b[43mstudent\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    410\u001b[39m \u001b[43m        \u001b[49m\u001b[43mteacher\u001b[49m\u001b[43m=\u001b[49m\u001b[43mteacher\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    411\u001b[39m \u001b[43m        \u001b[49m\u001b[43mtrainset\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtrainset_copy\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    412\u001b[39m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    414\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m i, _ \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(student.predictors()):\n\u001b[32m    415\u001b[39m     demo_candidates[i].append(program2.predictors()[i].demos)\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/dspy/teleprompt/bootstrap.py:86\u001b[39m, in \u001b[36mBootstrapFewShot.compile\u001b[39m\u001b[34m(self, student, teacher, trainset)\u001b[39m\n\u001b[32m     84\u001b[39m \u001b[38;5;28mself\u001b[39m._prepare_student_and_teacher(student, teacher)\n\u001b[32m     85\u001b[39m \u001b[38;5;28mself\u001b[39m._prepare_predictor_mappings()\n\u001b[32m---> \u001b[39m\u001b[32m86\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_bootstrap\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m     88\u001b[39m \u001b[38;5;28mself\u001b[39m.student = \u001b[38;5;28mself\u001b[39m._train()\n\u001b[32m     89\u001b[39m \u001b[38;5;28mself\u001b[39m.student._compiled = \u001b[38;5;28;01mTrue\u001b[39;00m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/dspy/teleprompt/bootstrap.py:159\u001b[39m, in \u001b[36mBootstrapFewShot._bootstrap\u001b[39m\u001b[34m(self, max_bootstraps)\u001b[39m\n\u001b[32m    156\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m round_idx \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;28mself\u001b[39m.max_rounds):\n\u001b[32m    157\u001b[39m     bootstrap_attempts += \u001b[32m1\u001b[39m\n\u001b[32m--> \u001b[39m\u001b[32m159\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_bootstrap_one_example\u001b[49m\u001b[43m(\u001b[49m\u001b[43mexample\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mround_idx\u001b[49m\u001b[43m)\u001b[49m:\n\u001b[32m    160\u001b[39m         bootstrapped[example_idx] = \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[32m    161\u001b[39m         \u001b[38;5;28;01mbreak\u001b[39;00m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/dspy/teleprompt/bootstrap.py:196\u001b[39m, in \u001b[36mBootstrapFewShot._bootstrap_one_example\u001b[39m\u001b[34m(self, example, round_idx)\u001b[39m\n\u001b[32m    193\u001b[39m     predictor_cache[name] = predictor.demos\n\u001b[32m    194\u001b[39m     predictor.demos = [x \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m predictor.demos \u001b[38;5;28;01mif\u001b[39;00m x != example]\n\u001b[32m--> \u001b[39m\u001b[32m196\u001b[39m prediction = \u001b[43mteacher\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mexample\u001b[49m\u001b[43m.\u001b[49m\u001b[43minputs\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    197\u001b[39m trace = dspy.settings.trace\n\u001b[32m    199\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m name, predictor \u001b[38;5;129;01min\u001b[39;00m teacher.named_predictors():\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/dspy/utils/callback.py:326\u001b[39m, in \u001b[36mwith_callbacks.<locals>.sync_wrapper\u001b[39m\u001b[34m(instance, *args, **kwargs)\u001b[39m\n\u001b[32m    324\u001b[39m callbacks = _get_active_callbacks(instance)\n\u001b[32m    325\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m callbacks:\n\u001b[32m--> \u001b[39m\u001b[32m326\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[43minstance\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    328\u001b[39m call_id = uuid.uuid4().hex\n\u001b[32m    330\u001b[39m _execute_start_callbacks(instance, fn, call_id, callbacks, args, kwargs)\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/dspy/primitives/module.py:78\u001b[39m, in \u001b[36mModule.__call__\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m     75\u001b[39m     output.set_lm_usage(usage_tracker.get_total_tokens())\n\u001b[32m     76\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m output\n\u001b[32m---> \u001b[39m\u001b[32m78\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mforward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/notebooks/../langwatch_nlp/studio/dspy/langwatch_workflow_module.py:112\u001b[39m, in \u001b[36mLangWatchWorkflowModule.prevent_crashes.<locals>.prevent_crashes_forward\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m    109\u001b[39m \u001b[38;5;129m@functools\u001b[39m.wraps(\u001b[38;5;28mself\u001b[39m.__forward_before_prevent_crashes__)\n\u001b[32m    110\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mprevent_crashes_forward\u001b[39m(\u001b[38;5;28mself\u001b[39m, *args, **kwargs):\n\u001b[32m    111\u001b[39m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m112\u001b[39m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m__forward_before_prevent_crashes__\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# type: ignore\u001b[39;00m\n\u001b[32m    113\u001b[39m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m    114\u001b[39m         \u001b[38;5;28;01mreturn\u001b[39;00m PredictionWithEvaluationAndMetadata(\n\u001b[32m    115\u001b[39m             duration=\u001b[38;5;28mself\u001b[39m.duration,\n\u001b[32m    116\u001b[39m             cost=\u001b[38;5;28mself\u001b[39m.cost,\n\u001b[32m    117\u001b[39m             error=e,\n\u001b[32m    118\u001b[39m         )\n",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[4]\u001b[39m\u001b[32m, line 100\u001b[39m, in \u001b[36mWorkflowModule.forward\u001b[39m\u001b[34m(self, **kwargs)\u001b[39m\n\u001b[32m     97\u001b[39m \u001b[38;5;28mself\u001b[39m.cost = \u001b[32m0\u001b[39m\n\u001b[32m     98\u001b[39m \u001b[38;5;28mself\u001b[39m.duration = \u001b[32m0\u001b[39m\n\u001b[32m--> \u001b[39m\u001b[32m100\u001b[39m classify_transaction = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mclassify_transaction\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m    101\u001b[39m \u001b[43m    \u001b[49m\u001b[43mdescription\u001b[49m\u001b[43m=\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mdescription\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    102\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    104\u001b[39m exact_match = \u001b[38;5;28mself\u001b[39m.exact_match(\n\u001b[32m    105\u001b[39m     data={\n\u001b[32m    106\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33moutput\u001b[39m\u001b[33m\"\u001b[39m: classify_transaction.category,\n\u001b[32m    107\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33mexpected_output\u001b[39m\u001b[33m\"\u001b[39m: kwargs.get(\u001b[33m\"\u001b[39m\u001b[33mcategory\u001b[39m\u001b[33m\"\u001b[39m),\n\u001b[32m    108\u001b[39m     }\n\u001b[32m    109\u001b[39m )\n\u001b[32m    111\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m PredictionWithEvaluationAndMetadata(\n\u001b[32m    112\u001b[39m     classify_transaction=classify_transaction,\n\u001b[32m    113\u001b[39m     end={\n\u001b[32m   (...)\u001b[39m\u001b[32m    120\u001b[39m     duration=\u001b[38;5;28mself\u001b[39m.duration,\n\u001b[32m    121\u001b[39m )\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/dspy/utils/callback.py:326\u001b[39m, in \u001b[36mwith_callbacks.<locals>.sync_wrapper\u001b[39m\u001b[34m(instance, *args, **kwargs)\u001b[39m\n\u001b[32m    324\u001b[39m callbacks = _get_active_callbacks(instance)\n\u001b[32m    325\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m callbacks:\n\u001b[32m--> \u001b[39m\u001b[32m326\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[43minstance\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    328\u001b[39m call_id = uuid.uuid4().hex\n\u001b[32m    330\u001b[39m _execute_start_callbacks(instance, fn, call_id, callbacks, args, kwargs)\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/dspy/primitives/module.py:78\u001b[39m, in \u001b[36mModule.__call__\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m     75\u001b[39m     output.set_lm_usage(usage_tracker.get_total_tokens())\n\u001b[32m     76\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m output\n\u001b[32m---> \u001b[39m\u001b[32m78\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mforward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/notebooks/../langwatch_nlp/studio/dspy/langwatch_workflow_module.py:55\u001b[39m, in \u001b[36mLangWatchWorkflowModule.wrapped.<locals>.forward_with_metadata\u001b[39m\u001b[34m(instance_self, *args, **kwargs)\u001b[39m\n\u001b[32m     53\u001b[39m start_time = time.time()\n\u001b[32m     54\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m55\u001b[39m     result = \u001b[43mmodule\u001b[49m\u001b[43m.\u001b[49m\u001b[43m__forward_before_metadata__\u001b[49m\u001b[43m(\u001b[49m\u001b[43minstance_self\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# type: ignore\u001b[39;00m\n\u001b[32m     56\u001b[39m     \u001b[38;5;66;03m# Skip cost and duration calculation for evaluation results as those are counted separately\u001b[39;00m\n\u001b[32m     57\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(result, PredictionWithEvaluationAndMetadata):\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/notebooks/../langwatch_nlp/studio/dspy/reporting_module.py:57\u001b[39m, in \u001b[36mReportingModule.with_reporting.<locals>.forward_with_reporting\u001b[39m\u001b[34m(instance_self, *args, **kwargs)\u001b[39m\n\u001b[32m     53\u001b[39m     \u001b[38;5;28mself\u001b[39m.context.queue.put_nowait(\n\u001b[32m     54\u001b[39m         start_component_event(node, \u001b[38;5;28mself\u001b[39m.context.trace_id, kwargs)\n\u001b[32m     55\u001b[39m     )\n\u001b[32m     56\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m57\u001b[39m     result = \u001b[43mmodule\u001b[49m\u001b[43m.\u001b[49m\u001b[43m__forward_before_reporting__\u001b[49m\u001b[43m(\u001b[49m\u001b[43minstance_self\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# type: ignore\u001b[39;00m\n\u001b[32m     58\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m     59\u001b[39m     \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mtraceback\u001b[39;00m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/notebooks/../langwatch_nlp/studio/field_parser.py:178\u001b[39m, in \u001b[36mwith_autoparsing.<locals>.forward_with_autoparsing\u001b[39m\u001b[34m(instance_self, *args, **kwargs)\u001b[39m\n\u001b[32m    175\u001b[39m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m    176\u001b[39m         parsed_kwargs[key] = value\n\u001b[32m--> \u001b[39m\u001b[32m178\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward\u001b[49m\u001b[43m(\u001b[49m\u001b[43minstance_self\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43mparsed_args\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mparsed_kwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[4]\u001b[39m\u001b[32m, line 67\u001b[39m, in \u001b[36mClassifyTransaction.forward\u001b[39m\u001b[34m(self, description)\u001b[39m\n\u001b[32m     66\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, description: \u001b[38;5;28mstr\u001b[39m):\n\u001b[32m---> \u001b[39m\u001b[32m67\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m.\u001b[49m\u001b[43mforward\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdescription\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdescription\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/notebooks/../langwatch_nlp/studio/dspy/llm_node.py:39\u001b[39m, in \u001b[36mLLMNode.forward\u001b[39m\u001b[34m(self, **kwargs)\u001b[39m\n\u001b[32m     36\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m:\n\u001b[32m     37\u001b[39m     \u001b[38;5;28;01mpass\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m39\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mpredict\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/dspy/predict/predict.py:103\u001b[39m, in \u001b[36mPredict.__call__\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m    100\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m args:\n\u001b[32m    101\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;28mself\u001b[39m._get_positional_args_error_message())\n\u001b[32m--> \u001b[39m\u001b[32m103\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m.\u001b[49m\u001b[34;43m__call__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/dspy/utils/callback.py:326\u001b[39m, in \u001b[36mwith_callbacks.<locals>.sync_wrapper\u001b[39m\u001b[34m(instance, *args, **kwargs)\u001b[39m\n\u001b[32m    324\u001b[39m callbacks = _get_active_callbacks(instance)\n\u001b[32m    325\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m callbacks:\n\u001b[32m--> \u001b[39m\u001b[32m326\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[43minstance\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    328\u001b[39m call_id = uuid.uuid4().hex\n\u001b[32m    330\u001b[39m _execute_start_callbacks(instance, fn, call_id, callbacks, args, kwargs)\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/dspy/primitives/module.py:78\u001b[39m, in \u001b[36mModule.__call__\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m     75\u001b[39m     output.set_lm_usage(usage_tracker.get_total_tokens())\n\u001b[32m     76\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m output\n\u001b[32m---> \u001b[39m\u001b[32m78\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mforward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/notebooks/../langwatch_nlp/studio/dspy/predict_with_metadata.py:71\u001b[39m, in \u001b[36mPredictWithMetadata.forward\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m     69\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, *args, **kwargs):\n\u001b[32m     70\u001b[39m     \u001b[38;5;28mself\u001b[39m._module = \u001b[38;5;28msuper\u001b[39m().forward\n\u001b[32m---> \u001b[39m\u001b[32m71\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mModuleWithMetadata\u001b[49m\u001b[43m.\u001b[49m\u001b[43mforward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/notebooks/../langwatch_nlp/studio/dspy/predict_with_metadata.py:45\u001b[39m, in \u001b[36mModuleWithMetadata.forward\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m     43\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, *args, **kwargs):\n\u001b[32m     44\u001b[39m     start_time = time.time()\n\u001b[32m---> \u001b[39m\u001b[32m45\u001b[39m     response = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_module\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m     46\u001b[39m     duration = \u001b[38;5;28mround\u001b[39m((time.time() - start_time) * \u001b[32m1000\u001b[39m)\n\u001b[32m     48\u001b[39m     lm = cast(dspy.LM, \u001b[38;5;28mself\u001b[39m.get_lm()) \u001b[38;5;129;01mor\u001b[39;00m dspy.settings.lm\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/dspy/predict/predict.py:192\u001b[39m, in \u001b[36mPredict.forward\u001b[39m\u001b[34m(self, **kwargs)\u001b[39m\n\u001b[32m    190\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m    191\u001b[39m     \u001b[38;5;28;01mwith\u001b[39;00m settings.context(send_stream=\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[32m--> \u001b[39m\u001b[32m192\u001b[39m         completions = \u001b[43madapter\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlm\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlm_kwargs\u001b[49m\u001b[43m=\u001b[49m\u001b[43mconfig\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msignature\u001b[49m\u001b[43m=\u001b[49m\u001b[43msignature\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdemos\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdemos\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m=\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    194\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._forward_postprocess(completions, signature, **kwargs)\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/notebooks/../langwatch_nlp/studio/dspy/template_adapter.py:41\u001b[39m, in \u001b[36mTemplateAdapter.__call__\u001b[39m\u001b[34m(self, lm, lm_kwargs, signature, demos, inputs)\u001b[39m\n\u001b[32m     39\u001b[39m \u001b[38;5;66;03m# If the signature has only one output field and it's a string, we can use the text only completion\u001b[39;00m\n\u001b[32m     40\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m._use_text_only_completion(signature, inputs):\n\u001b[32m---> \u001b[39m\u001b[32m41\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mChatAdapter\u001b[49m\u001b[43m.\u001b[49m\u001b[34;43m__call__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlm\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlm_kwargs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msignature\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdemos\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# type: ignore\u001b[39;00m\n\u001b[32m     43\u001b[39m \u001b[38;5;66;03m# Replace the DSPyProgramOutputs title from the json schema with the signature name to bias the LLM in the right direction instead of randomly towards DSPy\u001b[39;00m\n\u001b[32m     44\u001b[39m model = _get_structured_outputs_response_format(signature)\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/dspy/adapters/chat_adapter.py:38\u001b[39m, in \u001b[36mChatAdapter.__call__\u001b[39m\u001b[34m(self, lm, lm_kwargs, signature, demos, inputs)\u001b[39m\n\u001b[32m     29\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34m__call__\u001b[39m(\n\u001b[32m     30\u001b[39m     \u001b[38;5;28mself\u001b[39m,\n\u001b[32m     31\u001b[39m     lm: LM,\n\u001b[32m   (...)\u001b[39m\u001b[32m     35\u001b[39m     inputs: \u001b[38;5;28mdict\u001b[39m[\u001b[38;5;28mstr\u001b[39m, Any],\n\u001b[32m     36\u001b[39m ) -> \u001b[38;5;28mlist\u001b[39m[\u001b[38;5;28mdict\u001b[39m[\u001b[38;5;28mstr\u001b[39m, Any]]:\n\u001b[32m     37\u001b[39m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m38\u001b[39m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m.\u001b[49m\u001b[34;43m__call__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mlm\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlm_kwargs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msignature\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdemos\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m     39\u001b[39m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m     40\u001b[39m         \u001b[38;5;66;03m# fallback to JSONAdapter\u001b[39;00m\n\u001b[32m     41\u001b[39m         \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mdspy\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01madapters\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mjson_adapter\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m JSONAdapter\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/dspy/adapters/base.py:127\u001b[39m, in \u001b[36mAdapter.__call__\u001b[39m\u001b[34m(self, lm, lm_kwargs, signature, demos, inputs)\u001b[39m\n\u001b[32m    124\u001b[39m processed_signature = \u001b[38;5;28mself\u001b[39m._call_preprocess(lm, lm_kwargs, signature, inputs)\n\u001b[32m    125\u001b[39m inputs = \u001b[38;5;28mself\u001b[39m.format(processed_signature, demos, inputs)\n\u001b[32m--> \u001b[39m\u001b[32m127\u001b[39m outputs = \u001b[43mlm\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmessages\u001b[49m\u001b[43m=\u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mlm_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    128\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._call_postprocess(processed_signature, signature, outputs)\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/python-sdk/src/langwatch/dspy/__init__.py:268\u001b[39m, in \u001b[36mLangWatchDSPy.patch_llms.<locals>.patched_call\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m    266\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mpatched_call\u001b[39m(\u001b[38;5;28mself\u001b[39m, *args, **kwargs):\n\u001b[32m    267\u001b[39m     classname = \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m.\u001b[34m__class__\u001b[39m.\u001b[34m__module__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m.\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m.\u001b[34m__class__\u001b[39m.\u001b[34m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m\n\u001b[32m--> \u001b[39m\u001b[32m268\u001b[39m     outputs = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_original_call\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    269\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m.history) > \u001b[32m0\u001b[39m:\n\u001b[32m    270\u001b[39m         entry = \u001b[38;5;28mself\u001b[39m.history[-\u001b[32m1\u001b[39m]\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/dspy/utils/callback.py:326\u001b[39m, in \u001b[36mwith_callbacks.<locals>.sync_wrapper\u001b[39m\u001b[34m(instance, *args, **kwargs)\u001b[39m\n\u001b[32m    324\u001b[39m callbacks = _get_active_callbacks(instance)\n\u001b[32m    325\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m callbacks:\n\u001b[32m--> \u001b[39m\u001b[32m326\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[43minstance\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    328\u001b[39m call_id = uuid.uuid4().hex\n\u001b[32m    330\u001b[39m _execute_start_callbacks(instance, fn, call_id, callbacks, args, kwargs)\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/dspy/clients/base_lm.py:85\u001b[39m, in \u001b[36mBaseLM.__call__\u001b[39m\u001b[34m(self, prompt, messages, **kwargs)\u001b[39m\n\u001b[32m     83\u001b[39m \u001b[38;5;129m@with_callbacks\u001b[39m\n\u001b[32m     84\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, prompt=\u001b[38;5;28;01mNone\u001b[39;00m, messages=\u001b[38;5;28;01mNone\u001b[39;00m, **kwargs):\n\u001b[32m---> \u001b[39m\u001b[32m85\u001b[39m     response = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mforward\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprompt\u001b[49m\u001b[43m=\u001b[49m\u001b[43mprompt\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmessages\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmessages\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m     86\u001b[39m     outputs = \u001b[38;5;28mself\u001b[39m._process_lm_response(response, prompt, messages, **kwargs)\n\u001b[32m     88\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m outputs\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/dspy/clients/lm.py:147\u001b[39m, in \u001b[36mLM.forward\u001b[39m\u001b[34m(self, prompt, messages, **kwargs)\u001b[39m\n\u001b[32m    144\u001b[39m     completion = litellm_responses_completion\n\u001b[32m    145\u001b[39m completion, litellm_cache_args = \u001b[38;5;28mself\u001b[39m._get_cached_completion_fn(completion, cache)\n\u001b[32m--> \u001b[39m\u001b[32m147\u001b[39m results = \u001b[43mcompletion\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m    148\u001b[39m \u001b[43m    \u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mdict\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmessages\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmessages\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    149\u001b[39m \u001b[43m    \u001b[49m\u001b[43mnum_retries\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mnum_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    150\u001b[39m \u001b[43m    \u001b[49m\u001b[43mcache\u001b[49m\u001b[43m=\u001b[49m\u001b[43mlitellm_cache_args\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    151\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    153\u001b[39m \u001b[38;5;28mself\u001b[39m._check_truncation(results)\n\u001b[32m    155\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mgetattr\u001b[39m(results, \u001b[33m\"\u001b[39m\u001b[33mcache_hit\u001b[39m\u001b[33m\"\u001b[39m, \u001b[38;5;28;01mFalse\u001b[39;00m) \u001b[38;5;129;01mand\u001b[39;00m dspy.settings.usage_tracker \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(results, \u001b[33m\"\u001b[39m\u001b[33musage\u001b[39m\u001b[33m\"\u001b[39m):\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/dspy/clients/cache.py:235\u001b[39m, in \u001b[36mrequest_cache.<locals>.decorator.<locals>.sync_wrapper\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m    232\u001b[39m \u001b[38;5;66;03m# Otherwise, compute and store the result\u001b[39;00m\n\u001b[32m    233\u001b[39m \u001b[38;5;66;03m# Make a copy of the original request in case it's modified in place, e.g., deleting some fields\u001b[39;00m\n\u001b[32m    234\u001b[39m original_request = copy.deepcopy(modified_request)\n\u001b[32m--> \u001b[39m\u001b[32m235\u001b[39m result = \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    236\u001b[39m \u001b[38;5;66;03m# `enable_memory_cache` can be provided at call time to avoid indefinite growth.\u001b[39;00m\n\u001b[32m    237\u001b[39m cache.put(original_request, result, ignored_args_for_cache_key, enable_memory_cache)\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/dspy/clients/lm.py:336\u001b[39m, in \u001b[36mlitellm_completion\u001b[39m\u001b[34m(request, num_retries, cache)\u001b[39m\n\u001b[32m    334\u001b[39m stream_completion = _get_stream_completion_fn(request, cache, sync=\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[32m    335\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m stream_completion \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m336\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mlitellm\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcompletion\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m    337\u001b[39m \u001b[43m        \u001b[49m\u001b[43mcache\u001b[49m\u001b[43m=\u001b[49m\u001b[43mcache\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    338\u001b[39m \u001b[43m        \u001b[49m\u001b[43mnum_retries\u001b[49m\u001b[43m=\u001b[49m\u001b[43mnum_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    339\u001b[39m \u001b[43m        \u001b[49m\u001b[43mretry_strategy\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mexponential_backoff_retry\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m    340\u001b[39m \u001b[43m        \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    341\u001b[39m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    343\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m stream_completion()\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/langevals_core/litellm_patch.py:72\u001b[39m, in \u001b[36mpatch_litellm.<locals>.patched_completion\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m     69\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mpatched_completion\u001b[39m(*args, **kwargs):\n\u001b[32m     70\u001b[39m     kwargs = patch_litellm_params(kwargs)\n\u001b[32m---> \u001b[39m\u001b[32m72\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_original_completion\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/litellm/utils.py:1244\u001b[39m, in \u001b[36mclient.<locals>.wrapper\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m   1242\u001b[39m         print_verbose(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mError while checking max token limit: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mstr\u001b[39m(e)\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m)\n\u001b[32m   1243\u001b[39m \u001b[38;5;66;03m# MODEL CALL\u001b[39;00m\n\u001b[32m-> \u001b[39m\u001b[32m1244\u001b[39m result = \u001b[43moriginal_function\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m   1245\u001b[39m end_time = datetime.datetime.now()\n\u001b[32m   1246\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m _is_streaming_request(\n\u001b[32m   1247\u001b[39m     kwargs=kwargs,\n\u001b[32m   1248\u001b[39m     call_type=call_type,\n\u001b[32m   1249\u001b[39m ):\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/litellm/main.py:2125\u001b[39m, in \u001b[36mcompletion\u001b[39m\u001b[34m(model, messages, timeout, temperature, top_p, n, stream, stream_options, stop, max_completion_tokens, max_tokens, modalities, prediction, audio, presence_penalty, frequency_penalty, logit_bias, user, reasoning_effort, response_format, seed, tools, tool_choice, logprobs, top_logprobs, parallel_tool_calls, web_search_options, deployment_id, extra_headers, safety_identifier, service_tier, functions, function_call, base_url, api_version, api_key, model_list, thinking, shared_session, **kwargs)\u001b[39m\n\u001b[32m   2105\u001b[39m         response = base_llm_http_handler.completion(\n\u001b[32m   2106\u001b[39m             model=model,\n\u001b[32m   2107\u001b[39m             messages=messages,\n\u001b[32m   (...)\u001b[39m\u001b[32m   2122\u001b[39m             provider_config=provider_config,\n\u001b[32m   2123\u001b[39m         )\n\u001b[32m   2124\u001b[39m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m2125\u001b[39m         response = \u001b[43mopenai_chat_completions\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcompletion\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m   2126\u001b[39m \u001b[43m            \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   2127\u001b[39m \u001b[43m            \u001b[49m\u001b[43mmessages\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmessages\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   2128\u001b[39m \u001b[43m            \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   2129\u001b[39m \u001b[43m            \u001b[49m\u001b[43mmodel_response\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmodel_response\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   2130\u001b[39m \u001b[43m            \u001b[49m\u001b[43mprint_verbose\u001b[49m\u001b[43m=\u001b[49m\u001b[43mprint_verbose\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   2131\u001b[39m \u001b[43m            \u001b[49m\u001b[43mapi_key\u001b[49m\u001b[43m=\u001b[49m\u001b[43mapi_key\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   2132\u001b[39m \u001b[43m            \u001b[49m\u001b[43mapi_base\u001b[49m\u001b[43m=\u001b[49m\u001b[43mapi_base\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   2133\u001b[39m \u001b[43m            \u001b[49m\u001b[43macompletion\u001b[49m\u001b[43m=\u001b[49m\u001b[43macompletion\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   2134\u001b[39m \u001b[43m            \u001b[49m\u001b[43mlogging_obj\u001b[49m\u001b[43m=\u001b[49m\u001b[43mlogging\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   2135\u001b[39m \u001b[43m            \u001b[49m\u001b[43moptional_params\u001b[49m\u001b[43m=\u001b[49m\u001b[43moptional_params\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   2136\u001b[39m \u001b[43m            \u001b[49m\u001b[43mlitellm_params\u001b[49m\u001b[43m=\u001b[49m\u001b[43mlitellm_params\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   2137\u001b[39m \u001b[43m            \u001b[49m\u001b[43mlogger_fn\u001b[49m\u001b[43m=\u001b[49m\u001b[43mlogger_fn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   2138\u001b[39m \u001b[43m            \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# type: ignore\u001b[39;49;00m\n\u001b[32m   2139\u001b[39m \u001b[43m            \u001b[49m\u001b[43mcustom_prompt_dict\u001b[49m\u001b[43m=\u001b[49m\u001b[43mcustom_prompt_dict\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   2140\u001b[39m \u001b[43m            \u001b[49m\u001b[43mclient\u001b[49m\u001b[43m=\u001b[49m\u001b[43mclient\u001b[49m\u001b[43m,\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# pass AsyncOpenAI, OpenAI client\u001b[39;49;00m\n\u001b[32m   2141\u001b[39m \u001b[43m            \u001b[49m\u001b[43morganization\u001b[49m\u001b[43m=\u001b[49m\u001b[43morganization\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   2142\u001b[39m \u001b[43m            \u001b[49m\u001b[43mcustom_llm_provider\u001b[49m\u001b[43m=\u001b[49m\u001b[43mcustom_llm_provider\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   2143\u001b[39m \u001b[43m            \u001b[49m\u001b[43mshared_session\u001b[49m\u001b[43m=\u001b[49m\u001b[43mshared_session\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   2144\u001b[39m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m   2145\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m   2146\u001b[39m     \u001b[38;5;66;03m## LOGGING - log the original exception returned\u001b[39;00m\n\u001b[32m   2147\u001b[39m     logging.post_call(\n\u001b[32m   2148\u001b[39m         \u001b[38;5;28minput\u001b[39m=messages,\n\u001b[32m   2149\u001b[39m         api_key=api_key,\n\u001b[32m   2150\u001b[39m         original_response=\u001b[38;5;28mstr\u001b[39m(e),\n\u001b[32m   2151\u001b[39m         additional_args={\u001b[33m\"\u001b[39m\u001b[33mheaders\u001b[39m\u001b[33m\"\u001b[39m: headers},\n\u001b[32m   2152\u001b[39m     )\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/litellm/llms/openai/openai.py:673\u001b[39m, in \u001b[36mOpenAIChatCompletion.completion\u001b[39m\u001b[34m(self, model_response, timeout, optional_params, litellm_params, logging_obj, model, messages, print_verbose, api_key, api_base, api_version, dynamic_params, azure_ad_token, acompletion, logger_fn, headers, custom_prompt_dict, client, organization, custom_llm_provider, drop_params, shared_session)\u001b[39m\n\u001b[32m    658\u001b[39m \u001b[38;5;66;03m## LOGGING\u001b[39;00m\n\u001b[32m    659\u001b[39m logging_obj.pre_call(\n\u001b[32m    660\u001b[39m     \u001b[38;5;28minput\u001b[39m=messages,\n\u001b[32m    661\u001b[39m     api_key=openai_client.api_key,\n\u001b[32m   (...)\u001b[39m\u001b[32m    667\u001b[39m     },\n\u001b[32m    668\u001b[39m )\n\u001b[32m    670\u001b[39m (\n\u001b[32m    671\u001b[39m     headers,\n\u001b[32m    672\u001b[39m     response,\n\u001b[32m--> \u001b[39m\u001b[32m673\u001b[39m ) = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mmake_sync_openai_chat_completion_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m    674\u001b[39m \u001b[43m    \u001b[49m\u001b[43mopenai_client\u001b[49m\u001b[43m=\u001b[49m\u001b[43mopenai_client\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    675\u001b[39m \u001b[43m    \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    676\u001b[39m \u001b[43m    \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    677\u001b[39m \u001b[43m    \u001b[49m\u001b[43mlogging_obj\u001b[49m\u001b[43m=\u001b[49m\u001b[43mlogging_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    678\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    680\u001b[39m logging_obj.model_call_details[\u001b[33m\"\u001b[39m\u001b[33mresponse_headers\u001b[39m\u001b[33m\"\u001b[39m] = headers\n\u001b[32m    681\u001b[39m stringified_response = response.model_dump()\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/litellm/litellm_core_utils/logging_utils.py:237\u001b[39m, in \u001b[36mtrack_llm_api_timing.<locals>.decorator.<locals>.sync_wrapper\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m    234\u001b[39m parent_otel_span = _get_parent_otel_span_from_logging_obj(logging_obj)\n\u001b[32m    236\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m237\u001b[39m     result = \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    238\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m result\n\u001b[32m    239\u001b[39m \u001b[38;5;28;01mfinally\u001b[39;00m:\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/litellm/llms/openai/openai.py:471\u001b[39m, in \u001b[36mOpenAIChatCompletion.make_sync_openai_chat_completion_request\u001b[39m\u001b[34m(self, openai_client, data, timeout, logging_obj)\u001b[39m\n\u001b[32m    469\u001b[39m raw_response = \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m    470\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m471\u001b[39m     raw_response = \u001b[43mopenai_client\u001b[49m\u001b[43m.\u001b[49m\u001b[43mchat\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcompletions\u001b[49m\u001b[43m.\u001b[49m\u001b[43mwith_raw_response\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcreate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m    472\u001b[39m \u001b[43m        \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\n\u001b[32m    473\u001b[39m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    475\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(raw_response, \u001b[33m\"\u001b[39m\u001b[33mheaders\u001b[39m\u001b[33m\"\u001b[39m):\n\u001b[32m    476\u001b[39m         headers = \u001b[38;5;28mdict\u001b[39m(raw_response.headers)\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/openai/_legacy_response.py:364\u001b[39m, in \u001b[36mto_raw_response_wrapper.<locals>.wrapped\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m    360\u001b[39m extra_headers[RAW_RESPONSE_HEADER] = \u001b[33m\"\u001b[39m\u001b[33mtrue\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    362\u001b[39m kwargs[\u001b[33m\"\u001b[39m\u001b[33mextra_headers\u001b[39m\u001b[33m\"\u001b[39m] = extra_headers\n\u001b[32m--> \u001b[39m\u001b[32m364\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m cast(LegacyAPIResponse[R], \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m)\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/openai/_utils/_utils.py:286\u001b[39m, in \u001b[36mrequired_args.<locals>.inner.<locals>.wrapper\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m    284\u001b[39m             msg = \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mMissing required argument: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mquote(missing[\u001b[32m0\u001b[39m])\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m\n\u001b[32m    285\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(msg)\n\u001b[32m--> \u001b[39m\u001b[32m286\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/openai/resources/chat/completions/completions.py:1147\u001b[39m, in \u001b[36mCompletions.create\u001b[39m\u001b[34m(self, messages, model, audio, frequency_penalty, function_call, functions, logit_bias, logprobs, max_completion_tokens, max_tokens, metadata, modalities, n, parallel_tool_calls, prediction, presence_penalty, prompt_cache_key, reasoning_effort, response_format, safety_identifier, seed, service_tier, stop, store, stream, stream_options, temperature, tool_choice, tools, top_logprobs, top_p, user, verbosity, web_search_options, extra_headers, extra_query, extra_body, timeout)\u001b[39m\n\u001b[32m   1101\u001b[39m \u001b[38;5;129m@required_args\u001b[39m([\u001b[33m\"\u001b[39m\u001b[33mmessages\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mmodel\u001b[39m\u001b[33m\"\u001b[39m], [\u001b[33m\"\u001b[39m\u001b[33mmessages\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mmodel\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mstream\u001b[39m\u001b[33m\"\u001b[39m])\n\u001b[32m   1102\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mcreate\u001b[39m(\n\u001b[32m   1103\u001b[39m     \u001b[38;5;28mself\u001b[39m,\n\u001b[32m   (...)\u001b[39m\u001b[32m   1144\u001b[39m     timeout: \u001b[38;5;28mfloat\u001b[39m | httpx.Timeout | \u001b[38;5;28;01mNone\u001b[39;00m | NotGiven = not_given,\n\u001b[32m   1145\u001b[39m ) -> ChatCompletion | Stream[ChatCompletionChunk]:\n\u001b[32m   1146\u001b[39m     validate_response_format(response_format)\n\u001b[32m-> \u001b[39m\u001b[32m1147\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_post\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m   1148\u001b[39m \u001b[43m        \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m/chat/completions\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m   1149\u001b[39m \u001b[43m        \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmaybe_transform\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m   1150\u001b[39m \u001b[43m            \u001b[49m\u001b[43m{\u001b[49m\n\u001b[32m   1151\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mmessages\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mmessages\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1152\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mmodel\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1153\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43maudio\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43maudio\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1154\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mfrequency_penalty\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mfrequency_penalty\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1155\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mfunction_call\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mfunction_call\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1156\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mfunctions\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mfunctions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1157\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mlogit_bias\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mlogit_bias\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1158\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mlogprobs\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mlogprobs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1159\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mmax_completion_tokens\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mmax_completion_tokens\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1160\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mmax_tokens\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mmax_tokens\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1161\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mmetadata\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mmetadata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1162\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mmodalities\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodalities\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1163\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mn\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1164\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mparallel_tool_calls\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mparallel_tool_calls\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1165\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mprediction\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mprediction\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1166\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mpresence_penalty\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mpresence_penalty\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1167\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mprompt_cache_key\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mprompt_cache_key\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1168\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mreasoning_effort\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mreasoning_effort\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1169\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mresponse_format\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mresponse_format\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1170\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43msafety_identifier\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43msafety_identifier\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1171\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mseed\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mseed\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1172\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mservice_tier\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mservice_tier\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1173\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mstop\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mstop\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1174\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mstore\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mstore\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1175\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mstream\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1176\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mstream_options\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mstream_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1177\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mtemperature\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mtemperature\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1178\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mtool_choice\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mtool_choice\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1179\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mtools\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mtools\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1180\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mtop_logprobs\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mtop_logprobs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1181\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mtop_p\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mtop_p\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1182\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43muser\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43muser\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1183\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mverbosity\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mverbosity\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1184\u001b[39m \u001b[43m                \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mweb_search_options\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mweb_search_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1185\u001b[39m \u001b[43m            \u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1186\u001b[39m \u001b[43m            \u001b[49m\u001b[43mcompletion_create_params\u001b[49m\u001b[43m.\u001b[49m\u001b[43mCompletionCreateParamsStreaming\u001b[49m\n\u001b[32m   1187\u001b[39m \u001b[43m            \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\n\u001b[32m   1188\u001b[39m \u001b[43m            \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mcompletion_create_params\u001b[49m\u001b[43m.\u001b[49m\u001b[43mCompletionCreateParamsNonStreaming\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1189\u001b[39m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1190\u001b[39m \u001b[43m        \u001b[49m\u001b[43moptions\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmake_request_options\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m   1191\u001b[39m \u001b[43m            \u001b[49m\u001b[43mextra_headers\u001b[49m\u001b[43m=\u001b[49m\u001b[43mextra_headers\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mextra_query\u001b[49m\u001b[43m=\u001b[49m\u001b[43mextra_query\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mextra_body\u001b[49m\u001b[43m=\u001b[49m\u001b[43mextra_body\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\n\u001b[32m   1192\u001b[39m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1193\u001b[39m \u001b[43m        \u001b[49m\u001b[43mcast_to\u001b[49m\u001b[43m=\u001b[49m\u001b[43mChatCompletion\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1194\u001b[39m \u001b[43m        \u001b[49m\u001b[43mstream\u001b[49m\u001b[43m=\u001b[49m\u001b[43mstream\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m   1195\u001b[39m \u001b[43m        \u001b[49m\u001b[43mstream_cls\u001b[49m\u001b[43m=\u001b[49m\u001b[43mStream\u001b[49m\u001b[43m[\u001b[49m\u001b[43mChatCompletionChunk\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1196\u001b[39m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/openai/_base_client.py:1259\u001b[39m, in \u001b[36mSyncAPIClient.post\u001b[39m\u001b[34m(self, path, cast_to, body, options, files, stream, stream_cls)\u001b[39m\n\u001b[32m   1245\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mpost\u001b[39m(\n\u001b[32m   1246\u001b[39m     \u001b[38;5;28mself\u001b[39m,\n\u001b[32m   1247\u001b[39m     path: \u001b[38;5;28mstr\u001b[39m,\n\u001b[32m   (...)\u001b[39m\u001b[32m   1254\u001b[39m     stream_cls: \u001b[38;5;28mtype\u001b[39m[_StreamT] | \u001b[38;5;28;01mNone\u001b[39;00m = \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[32m   1255\u001b[39m ) -> ResponseT | _StreamT:\n\u001b[32m   1256\u001b[39m     opts = FinalRequestOptions.construct(\n\u001b[32m   1257\u001b[39m         method=\u001b[33m\"\u001b[39m\u001b[33mpost\u001b[39m\u001b[33m\"\u001b[39m, url=path, json_data=body, files=to_httpx_files(files), **options\n\u001b[32m   1258\u001b[39m     )\n\u001b[32m-> \u001b[39m\u001b[32m1259\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m cast(ResponseT, \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcast_to\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mopts\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[43m=\u001b[49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstream_cls\u001b[49m\u001b[43m=\u001b[49m\u001b[43mstream_cls\u001b[49m\u001b[43m)\u001b[49m)\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/openai/_base_client.py:982\u001b[39m, in \u001b[36mSyncAPIClient.request\u001b[39m\u001b[34m(self, cast_to, options, stream, stream_cls)\u001b[39m\n\u001b[32m    980\u001b[39m response = \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m    981\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m982\u001b[39m     response = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_client\u001b[49m\u001b[43m.\u001b[49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m    983\u001b[39m \u001b[43m        \u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    984\u001b[39m \u001b[43m        \u001b[49m\u001b[43mstream\u001b[49m\u001b[43m=\u001b[49m\u001b[43mstream\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_should_stream_response_body\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m=\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    985\u001b[39m \u001b[43m        \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    986\u001b[39m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    987\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m httpx.TimeoutException \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[32m    988\u001b[39m     log.debug(\u001b[33m\"\u001b[39m\u001b[33mEncountered httpx.TimeoutException\u001b[39m\u001b[33m\"\u001b[39m, exc_info=\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/httpx/_client.py:914\u001b[39m, in \u001b[36mClient.send\u001b[39m\u001b[34m(self, request, stream, auth, follow_redirects)\u001b[39m\n\u001b[32m    910\u001b[39m \u001b[38;5;28mself\u001b[39m._set_timeout(request)\n\u001b[32m    912\u001b[39m auth = \u001b[38;5;28mself\u001b[39m._build_request_auth(request, auth)\n\u001b[32m--> \u001b[39m\u001b[32m914\u001b[39m response = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_send_handling_auth\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m    915\u001b[39m \u001b[43m    \u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    916\u001b[39m \u001b[43m    \u001b[49m\u001b[43mauth\u001b[49m\u001b[43m=\u001b[49m\u001b[43mauth\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    917\u001b[39m \u001b[43m    \u001b[49m\u001b[43mfollow_redirects\u001b[49m\u001b[43m=\u001b[49m\u001b[43mfollow_redirects\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    918\u001b[39m \u001b[43m    \u001b[49m\u001b[43mhistory\u001b[49m\u001b[43m=\u001b[49m\u001b[43m[\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    919\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    920\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m    921\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m stream:\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/httpx/_client.py:942\u001b[39m, in \u001b[36mClient._send_handling_auth\u001b[39m\u001b[34m(self, request, auth, follow_redirects, history)\u001b[39m\n\u001b[32m    939\u001b[39m request = \u001b[38;5;28mnext\u001b[39m(auth_flow)\n\u001b[32m    941\u001b[39m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m942\u001b[39m     response = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_send_handling_redirects\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m    943\u001b[39m \u001b[43m        \u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    944\u001b[39m \u001b[43m        \u001b[49m\u001b[43mfollow_redirects\u001b[49m\u001b[43m=\u001b[49m\u001b[43mfollow_redirects\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    945\u001b[39m \u001b[43m        \u001b[49m\u001b[43mhistory\u001b[49m\u001b[43m=\u001b[49m\u001b[43mhistory\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    946\u001b[39m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    947\u001b[39m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m    948\u001b[39m         \u001b[38;5;28;01mtry\u001b[39;00m:\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/httpx/_client.py:979\u001b[39m, in \u001b[36mClient._send_handling_redirects\u001b[39m\u001b[34m(self, request, follow_redirects, history)\u001b[39m\n\u001b[32m    976\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m hook \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m._event_hooks[\u001b[33m\"\u001b[39m\u001b[33mrequest\u001b[39m\u001b[33m\"\u001b[39m]:\n\u001b[32m    977\u001b[39m     hook(request)\n\u001b[32m--> \u001b[39m\u001b[32m979\u001b[39m response = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_send_single_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    980\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m    981\u001b[39m     \u001b[38;5;28;01mfor\u001b[39;00m hook \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m._event_hooks[\u001b[33m\"\u001b[39m\u001b[33mresponse\u001b[39m\u001b[33m\"\u001b[39m]:\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/httpx/_client.py:1014\u001b[39m, in \u001b[36mClient._send_single_request\u001b[39m\u001b[34m(self, request)\u001b[39m\n\u001b[32m   1009\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[32m   1010\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33mAttempted to send an async request with a sync Client instance.\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m   1011\u001b[39m     )\n\u001b[32m   1013\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m request_context(request=request):\n\u001b[32m-> \u001b[39m\u001b[32m1014\u001b[39m     response = \u001b[43mtransport\u001b[49m\u001b[43m.\u001b[49m\u001b[43mhandle_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m   1016\u001b[39m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(response.stream, SyncByteStream)\n\u001b[32m   1018\u001b[39m response.request = request\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/httpx/_transports/default.py:250\u001b[39m, in \u001b[36mHTTPTransport.handle_request\u001b[39m\u001b[34m(self, request)\u001b[39m\n\u001b[32m    237\u001b[39m req = httpcore.Request(\n\u001b[32m    238\u001b[39m     method=request.method,\n\u001b[32m    239\u001b[39m     url=httpcore.URL(\n\u001b[32m   (...)\u001b[39m\u001b[32m    247\u001b[39m     extensions=request.extensions,\n\u001b[32m    248\u001b[39m )\n\u001b[32m    249\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m map_httpcore_exceptions():\n\u001b[32m--> \u001b[39m\u001b[32m250\u001b[39m     resp = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_pool\u001b[49m\u001b[43m.\u001b[49m\u001b[43mhandle_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    252\u001b[39m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(resp.stream, typing.Iterable)\n\u001b[32m    254\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m Response(\n\u001b[32m    255\u001b[39m     status_code=resp.status,\n\u001b[32m    256\u001b[39m     headers=resp.headers,\n\u001b[32m    257\u001b[39m     stream=ResponseStream(resp.stream),\n\u001b[32m    258\u001b[39m     extensions=resp.extensions,\n\u001b[32m    259\u001b[39m )\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/httpcore/_sync/connection_pool.py:256\u001b[39m, in \u001b[36mConnectionPool.handle_request\u001b[39m\u001b[34m(self, request)\u001b[39m\n\u001b[32m    253\u001b[39m         closing = \u001b[38;5;28mself\u001b[39m._assign_requests_to_connections()\n\u001b[32m    255\u001b[39m     \u001b[38;5;28mself\u001b[39m._close_connections(closing)\n\u001b[32m--> \u001b[39m\u001b[32m256\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m exc \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m    258\u001b[39m \u001b[38;5;66;03m# Return the response. Note that in this case we still have to manage\u001b[39;00m\n\u001b[32m    259\u001b[39m \u001b[38;5;66;03m# the point at which the response is closed.\u001b[39;00m\n\u001b[32m    260\u001b[39m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(response.stream, typing.Iterable)\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/httpcore/_sync/connection_pool.py:236\u001b[39m, in \u001b[36mConnectionPool.handle_request\u001b[39m\u001b[34m(self, request)\u001b[39m\n\u001b[32m    232\u001b[39m connection = pool_request.wait_for_connection(timeout=timeout)\n\u001b[32m    234\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m    235\u001b[39m     \u001b[38;5;66;03m# Send the request on the assigned connection.\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m236\u001b[39m     response = \u001b[43mconnection\u001b[49m\u001b[43m.\u001b[49m\u001b[43mhandle_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m    237\u001b[39m \u001b[43m        \u001b[49m\u001b[43mpool_request\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrequest\u001b[49m\n\u001b[32m    238\u001b[39m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    239\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m ConnectionNotAvailable:\n\u001b[32m    240\u001b[39m     \u001b[38;5;66;03m# In some cases a connection may initially be available to\u001b[39;00m\n\u001b[32m    241\u001b[39m     \u001b[38;5;66;03m# handle a request, but then become unavailable.\u001b[39;00m\n\u001b[32m    242\u001b[39m     \u001b[38;5;66;03m#\u001b[39;00m\n\u001b[32m    243\u001b[39m     \u001b[38;5;66;03m# In this case we clear the connection and try again.\u001b[39;00m\n\u001b[32m    244\u001b[39m     pool_request.clear_connection()\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/httpcore/_sync/connection.py:103\u001b[39m, in \u001b[36mHTTPConnection.handle_request\u001b[39m\u001b[34m(self, request)\u001b[39m\n\u001b[32m    100\u001b[39m     \u001b[38;5;28mself\u001b[39m._connect_failed = \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[32m    101\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m exc\n\u001b[32m--> \u001b[39m\u001b[32m103\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_connection\u001b[49m\u001b[43m.\u001b[49m\u001b[43mhandle_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/httpcore/_sync/http11.py:136\u001b[39m, in \u001b[36mHTTP11Connection.handle_request\u001b[39m\u001b[34m(self, request)\u001b[39m\n\u001b[32m    134\u001b[39m     \u001b[38;5;28;01mwith\u001b[39;00m Trace(\u001b[33m\"\u001b[39m\u001b[33mresponse_closed\u001b[39m\u001b[33m\"\u001b[39m, logger, request) \u001b[38;5;28;01mas\u001b[39;00m trace:\n\u001b[32m    135\u001b[39m         \u001b[38;5;28mself\u001b[39m._response_closed()\n\u001b[32m--> \u001b[39m\u001b[32m136\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m exc\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/httpcore/_sync/http11.py:106\u001b[39m, in \u001b[36mHTTP11Connection.handle_request\u001b[39m\u001b[34m(self, request)\u001b[39m\n\u001b[32m     95\u001b[39m     \u001b[38;5;28;01mpass\u001b[39;00m\n\u001b[32m     97\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m Trace(\n\u001b[32m     98\u001b[39m     \u001b[33m\"\u001b[39m\u001b[33mreceive_response_headers\u001b[39m\u001b[33m\"\u001b[39m, logger, request, kwargs\n\u001b[32m     99\u001b[39m ) \u001b[38;5;28;01mas\u001b[39;00m trace:\n\u001b[32m    100\u001b[39m     (\n\u001b[32m    101\u001b[39m         http_version,\n\u001b[32m    102\u001b[39m         status,\n\u001b[32m    103\u001b[39m         reason_phrase,\n\u001b[32m    104\u001b[39m         headers,\n\u001b[32m    105\u001b[39m         trailing_data,\n\u001b[32m--> \u001b[39m\u001b[32m106\u001b[39m     ) = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_receive_response_headers\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    107\u001b[39m     trace.return_value = (\n\u001b[32m    108\u001b[39m         http_version,\n\u001b[32m    109\u001b[39m         status,\n\u001b[32m    110\u001b[39m         reason_phrase,\n\u001b[32m    111\u001b[39m         headers,\n\u001b[32m    112\u001b[39m     )\n\u001b[32m    114\u001b[39m network_stream = \u001b[38;5;28mself\u001b[39m._network_stream\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/httpcore/_sync/http11.py:177\u001b[39m, in \u001b[36mHTTP11Connection._receive_response_headers\u001b[39m\u001b[34m(self, request)\u001b[39m\n\u001b[32m    174\u001b[39m timeout = timeouts.get(\u001b[33m\"\u001b[39m\u001b[33mread\u001b[39m\u001b[33m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[32m    176\u001b[39m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m177\u001b[39m     event = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_receive_event\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    178\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(event, h11.Response):\n\u001b[32m    179\u001b[39m         \u001b[38;5;28;01mbreak\u001b[39;00m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/httpcore/_sync/http11.py:217\u001b[39m, in \u001b[36mHTTP11Connection._receive_event\u001b[39m\u001b[34m(self, timeout)\u001b[39m\n\u001b[32m    214\u001b[39m     event = \u001b[38;5;28mself\u001b[39m._h11_state.next_event()\n\u001b[32m    216\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m event \u001b[38;5;129;01mis\u001b[39;00m h11.NEED_DATA:\n\u001b[32m--> \u001b[39m\u001b[32m217\u001b[39m     data = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_network_stream\u001b[49m\u001b[43m.\u001b[49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m    218\u001b[39m \u001b[43m        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mREAD_NUM_BYTES\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\n\u001b[32m    219\u001b[39m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    221\u001b[39m     \u001b[38;5;66;03m# If we feed this case through h11 we'll raise an exception like:\u001b[39;00m\n\u001b[32m    222\u001b[39m     \u001b[38;5;66;03m#\u001b[39;00m\n\u001b[32m    223\u001b[39m     \u001b[38;5;66;03m#     httpcore.RemoteProtocolError: can't handle event type\u001b[39;00m\n\u001b[32m   (...)\u001b[39m\u001b[32m    227\u001b[39m     \u001b[38;5;66;03m# perspective. Instead we handle this case distinctly and treat\u001b[39;00m\n\u001b[32m    228\u001b[39m     \u001b[38;5;66;03m# it as a ConnectError.\u001b[39;00m\n\u001b[32m    229\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m data == \u001b[33mb\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m\"\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m._h11_state.their_state == h11.SEND_RESPONSE:\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/langwatch-saas/langwatch/langwatch_nlp/.venv/lib/python3.12/site-packages/httpcore/_backends/sync.py:128\u001b[39m, in \u001b[36mSyncStream.read\u001b[39m\u001b[34m(self, max_bytes, timeout)\u001b[39m\n\u001b[32m    126\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m map_exceptions(exc_map):\n\u001b[32m    127\u001b[39m     \u001b[38;5;28mself\u001b[39m._sock.settimeout(timeout)\n\u001b[32m--> \u001b[39m\u001b[32m128\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_sock\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrecv\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmax_bytes\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/.pyenv/versions/3.12.0/lib/python3.12/ssl.py:1234\u001b[39m, in \u001b[36mSSLSocket.recv\u001b[39m\u001b[34m(self, buflen, flags)\u001b[39m\n\u001b[32m   1230\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m flags != \u001b[32m0\u001b[39m:\n\u001b[32m   1231\u001b[39m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[32m   1232\u001b[39m             \u001b[33m\"\u001b[39m\u001b[33mnon-zero flags not allowed in calls to recv() on \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[33m\"\u001b[39m %\n\u001b[32m   1233\u001b[39m             \u001b[38;5;28mself\u001b[39m.\u001b[34m__class__\u001b[39m)\n\u001b[32m-> \u001b[39m\u001b[32m1234\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbuflen\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m   1235\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m   1236\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28msuper\u001b[39m().recv(buflen, flags)\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/.pyenv/versions/3.12.0/lib/python3.12/ssl.py:1107\u001b[39m, in \u001b[36mSSLSocket.read\u001b[39m\u001b[34m(self, len, buffer)\u001b[39m\n\u001b[32m   1105\u001b[39m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._sslobj.read(\u001b[38;5;28mlen\u001b[39m, buffer)\n\u001b[32m   1106\u001b[39m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m1107\u001b[39m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_sslobj\u001b[49m\u001b[43m.\u001b[49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[32m   1108\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m SSLError \u001b[38;5;28;01mas\u001b[39;00m x:\n\u001b[32m   1109\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m x.args[\u001b[32m0\u001b[39m] == SSL_ERROR_EOF \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m.suppress_ragged_eofs:\n",
      "\u001b[31mKeyboardInterrupt\u001b[39m: "
     ]
    }
   ],
   "source": [
    "import dspy\n",
    "import sys\n",
    "import os\n",
    "\n",
    "from langwatch.dspy import LangWatchTrackedEvaluate\n",
    "\n",
    "from langwatch_nlp.studio.modules.evaluators.langwatch import LangWatchEvaluator\n",
    "\n",
    "sys.path.append(\"..\")\n",
    "\n",
    "from langwatch_nlp.studio.modules.evaluators.exact_match import ExactMatchEvaluator\n",
    "from langwatch_nlp.studio.dspy import (\n",
    "    LLMNode,\n",
    "    LangWatchWorkflowModule,\n",
    "    PredictionWithEvaluationAndMetadata,\n",
    "    TemplateAdapter,\n",
    ")\n",
    "import langwatch_nlp.studio.dspy.patched_caching\n",
    "\n",
    "# import langwatch_nlp.studio.dspy.patched_boostrap_few_shot\n",
    "import langwatch_nlp.studio.dspy.patched_optional_image\n",
    "\n",
    "\n",
    "class ClassifyTransactionSignature(dspy.Signature):\n",
    "    \"\"\"\"\"\"\n",
    "\n",
    "    _messages = [{\"role\": \"user\", \"content\": \"{{description}}\"}]\n",
    "    # _messages = []\n",
    "\n",
    "    description: str = dspy.InputField()\n",
    "    category: str = dspy.OutputField()\n",
    "\n",
    "\n",
    "class ClassifyTransaction(LLMNode):\n",
    "    def __init__(self):\n",
    "        predict = dspy.Predict(ClassifyTransactionSignature)\n",
    "\n",
    "        lm = dspy.LM(\"openai/gpt-4o-mini\")\n",
    "        demos = [\n",
    "            # {\"category\": \"Utilities\", \"description\": \"Power Company\"},\n",
    "            # {\"category\": \"Credit Card Payment\", \"description\": \"Credit Card Payment\"},\n",
    "            # {\"category\": \"Groceries\", \"description\": \"Grocery Store\"},\n",
    "            # {\"category\": \"Credit Card Payment\", \"description\": \"Credit Card Payment\"},\n",
    "            # {\"category\": \"Credit Card Payment\", \"description\": \"Credit Card Payment\"},\n",
    "            # {\"category\": \"Utilities\", \"description\": \"Gas Company\"},\n",
    "            # {\"category\": \"Electronics & Software\", \"description\": \"Best Buy\"},\n",
    "            # {\"category\": \"Restaurants\", \"description\": \"Thai Restaurant\"},\n",
    "            # {\"category\": \"Groceries\", \"description\": \"Grocery Store\"},\n",
    "            # {\"category\": \"Utilities\", \"description\": \"Gas Company\"},\n",
    "            # {\"category\": \"Restaurants\", \"description\": \"American Tavern\"},\n",
    "            # {\"category\": \"Shopping\", \"description\": \"Amazon\"},\n",
    "            # {\"category\": \"Coffee Shops\", \"description\": \"Starbucks\"},\n",
    "            # {\"category\": \"Credit Card Payment\", \"description\": \"Credit Card Payment\"},\n",
    "            # {\"category\": \"Gas & Fuel\", \"description\": \"Shell\"},\n",
    "            # {\"category\": \"Music\", \"description\": \"Spotify\"},\n",
    "        ]\n",
    "\n",
    "        super().__init__(\n",
    "            node_id=\"classify_transaction\",\n",
    "            name=\"ClassifyTransaction\",\n",
    "            predict=predict,\n",
    "            lm=lm,\n",
    "            demos=demos,\n",
    "        )\n",
    "\n",
    "    def forward(self, description: str):\n",
    "        return super().forward(description=description)\n",
    "\n",
    "\n",
    "# predictor = ClassifyTransaction()\n",
    "\n",
    "\n",
    "class ClassifyTransactionSimpler(dspy.Predict):\n",
    "    def forward(self, description: str):\n",
    "        return super().forward(description=description)\n",
    "\n",
    "\n",
    "class WorkflowModule(LangWatchWorkflowModule):\n",
    "    def __init__(self, run_evaluations: bool = False):\n",
    "        super().__init__()\n",
    "\n",
    "        self.exact_match = self.wrapped(\n",
    "            LangWatchEvaluator, node_id=\"exact_match\", run=run_evaluations\n",
    "        )(\n",
    "            api_key=os.environ[\"LANGWATCH_API_KEY\"],\n",
    "            name=\"ExactMatch\",\n",
    "            evaluator=\"langevals/exact_match\",\n",
    "            settings={},\n",
    "        )\n",
    "        self.classify_transaction = self.wrapped(\n",
    "            ClassifyTransaction,\n",
    "            node_id=\"classify_transaction\",\n",
    "        )()\n",
    "        # self.classify_transaction = ClassifyTransaction()\n",
    "\n",
    "    def forward(self, **kwargs) -> dspy.Prediction:\n",
    "        self.cost = 0\n",
    "        self.duration = 0\n",
    "\n",
    "        classify_transaction = self.classify_transaction(\n",
    "            description=kwargs.get(\"description\"),\n",
    "        )\n",
    "\n",
    "        exact_match = self.exact_match(\n",
    "            data={\n",
    "                \"output\": classify_transaction.category,\n",
    "                \"expected_output\": kwargs.get(\"category\"),\n",
    "            }\n",
    "        )\n",
    "\n",
    "        return PredictionWithEvaluationAndMetadata(\n",
    "            classify_transaction=classify_transaction,\n",
    "            end={\n",
    "                \"output\": classify_transaction.category,\n",
    "            },\n",
    "            evaluations={\n",
    "                \"exact_match\": exact_match,\n",
    "            },\n",
    "            cost=self.cost,\n",
    "            duration=self.duration,\n",
    "        )\n",
    "\n",
    "\n",
    "_original_compile = dspy.LabeledFewShot.compile\n",
    "\n",
    "\n",
    "class PatchedLabeledFewShot2(dspy.LabeledFewShot):\n",
    "    def compile(self, student, *, trainset, sample=True):\n",
    "        global map_labeled_examples\n",
    "\n",
    "        if not map_labeled_examples:\n",
    "            return _original_compile(self, student, trainset=trainset, sample=sample)\n",
    "\n",
    "        map_labeled_examples = False\n",
    "\n",
    "        self.student = student.reset_copy()\n",
    "        self.trainset = trainset\n",
    "\n",
    "        if len(self.trainset) == 0:\n",
    "            return self.student\n",
    "\n",
    "        rng = random.Random(0)\n",
    "\n",
    "        for predictor in self.student.predictors():\n",
    "            if not hasattr(predictor, \"_node_id\"):\n",
    "                continue\n",
    "\n",
    "            if sample:\n",
    "                samples = rng.sample(self.trainset, min(self.k, len(self.trainset)))\n",
    "            else:\n",
    "                samples = self.trainset[: min(self.k, len(self.trainset))]\n",
    "\n",
    "            samples = [demo.map_for_node(predictor._node_id) for demo in samples]\n",
    "            samples = [demo for demo in samples if demo is not None]\n",
    "            if len(samples) == 0:\n",
    "                continue\n",
    "\n",
    "            predictor.demos = samples\n",
    "\n",
    "        return self.student\n",
    "\n",
    "\n",
    "# predictor = ClassifyTransactionSimpler(ClassifyTransactionSignature)\n",
    "\n",
    "dspy.configure_cache(\n",
    "    enable_disk_cache=False,\n",
    "    # enable_memory_cache=False,\n",
    ")\n",
    "\n",
    "\n",
    "@langwatch.trace()\n",
    "def run():\n",
    "    with dspy.context(lm=dspy.LM(\"openai/gpt-4o-mini\"), adapter=TemplateAdapter()):\n",
    "        module = WorkflowModule(run_evaluations=True)\n",
    "        module.prevent_crashes()\n",
    "\n",
    "        pred = module(\n",
    "            description=\"say only 'Gas & Fuel', nothing else\", category=\"Gas & Fuel\"\n",
    "        )\n",
    "        print(pred, \"\\n\\n\\n\\n\")\n",
    "\n",
    "        def metric(\n",
    "            example: dspy.Example,\n",
    "            pred: PredictionWithEvaluationAndMetadata,\n",
    "            trace=None,\n",
    "        ):\n",
    "            score = pred.total_score(weighting=\"mean\")\n",
    "            return score\n",
    "\n",
    "        def metric(example, pred, trace=None):\n",
    "            return (\n",
    "                example.category.lower() == pred.classify_transaction.category.lower()\n",
    "            )\n",
    "\n",
    "        # module = dspy.Predict(ClassifyTransactionSignature)\n",
    "        # def metric(example, pred, trace=None):\n",
    "        #     return example.category.lower() == pred.category.lower()\n",
    "\n",
    "        optimizer = dspy.MIPROv2(\n",
    "            metric=metric,\n",
    "            auto=\"light\",\n",
    "            num_threads=24,\n",
    "        )\n",
    "\n",
    "        langwatch.dspy.init(\n",
    "            experiment=\"personal-transactions-from-notebook\", optimizer=optimizer\n",
    "        )\n",
    "\n",
    "        optimizer.compile(\n",
    "            module,\n",
    "            trainset=trainset_no_category_input,\n",
    "            valset=testset_no_category_input,\n",
    "            max_bootstrapped_demos=4,\n",
    "            max_labeled_demos=16,\n",
    "        )\n",
    "\n",
    "\n",
    "run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57b2c027",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
